#coding:utf-8

import json

import requests
import rss_log

from multiprocessing.dummy import Pool

ua = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.54 Safari/536.5'}

def fetch(u):
	try:
		r = requests.get(u,headers=ua,timeout=10)
		data = r.content
		return (data,u)
	except Exception,e:
		print e
		return (None,u)

def process(Us):
	print 'Num',len(Us)
	p = Pool(32)
	Ds = p.map(fetch, Us)
	p.close()

	def _base64(data):
		if data == None:
			return None
		else:
			return data.encode('base64')

	Ds = [ (_base64(data),u) for data,u in Ds ]

	Ds = [ (data,u) for data,u in Ds if data != None ]
	Ds = [ {'uri':u,'data':data} for data,u in Ds ]
	return Ds
